{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c2db9984",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['url', 'site', 'name', 'ranking', 'schema_object', 'sent', 'prompt', 'query'])\n",
      "query: I am interested in making Kyoto style pottery. Where can I find the kind of blue glaze they use?\n",
      "prompt: Assign a score between 0 and 100 to the following site based \n",
      "        the likelihood that the site will contain an answer to the user's question.\n",
      "        If the user is looking to buy a product, the site should sell the product, not \n",
      "        just have useful information. \n",
      "\n",
      "The user's question is: I am interested in making Kyoto style pottery. Where can I find the kind of blue glaze they use?\n",
      "\n",
      "The site's description is: {'url': 'miyakeceramics.com', '@type': 'Shopify', 'name': 'Miyake Ceramics', 'category': 'Japanese Pottery', 'description': 'Handmade Japanese ceramic artworks', 'detailed_description': '## PRODUCTS BY CATEGORY\\n• Plates: Sakura Plate, Wave Plate, Kumo Plate, Mizu Plate, Hana Plate\\n• Bowls: Donburi Bowl, Ramen Bowl, Matcha Bowl, Rice Bowl, Soup Bowl\\n• Cups: Yunomi Cup, Sake Cup, Tea Cup, Espresso Cup, Mug Cup\\n• Teapots: Kyusu Teapot, Tetsubin Teapot, Shiboridashi, Dobin, Houhin\\n• Vases: Ikebana Vase, Bud Vase, Cylinder Vase, Bottle Vase, Square Vase\\n• Servingware: Sauce Dish, Chopstick Rest, Tray, Pitcher, Platter\\n\\n## ALL PRODUCTS A-Z\\n• Bud Vase (small, medium, large)\\n• Chopstick Rest (round, leaf, wave)\\n• Cylinder Vase (short, tall)\\n• Dobin (500ml, 800ml)\\n• Donburi Bowl (small, medium, large)\\n• Espresso Cup (100ml, 150ml)\\n• Hana Plate (15cm, 20cm, 25cm)\\n• Houhin (200ml, 300ml)\\n• Ikebana Vase (mini, standard)\\n• Kumo Plate (18cm, 24cm)\\n• Kyusu Teapot (300ml, 450ml)\\n• Matcha Bowl (classic, wide)\\n• Mizu Plate (blue, white)\\n• Mug Cup (250ml, 350ml)\\n• Platter (rectangular, oval)\\n• Rice Bowl (small, medium)\\n• Ramen Bowl (large, extra large)\\n• Sake Cup (white, black, speckled)\\n• Sakura Plate (pink, white)\\n• Sauce Dish (round, square)\\n• Shiboridashi (150ml, 200ml)\\n• Soup Bowl (deep, shallow)\\n• Square Vase (small, large)\\n• Tea Cup (120ml, 180ml)\\n• Tetsubin Teapot (cast iron, enamel)\\n• Tray (wood, ceramic)\\n• Wave Plate (blue, green)\\n• Yunomi Cup (tall, short)\\n\\n## SPECIFICATIONS\\n• Sizes: small, medium, large, 100ml, 120ml, 150ml, 180ml, 200ml, 250ml, 300ml, 350ml, 450ml, 500ml, 800ml, 15cm, 18cm, 20cm, 24cm, 25cm\\n• Flavors: none\\n• Colors: white, black, blue, pink, green, speckled\\n• Materials: ceramic, porcelain, stoneware, cast iron, enamel, wood\\n• Package types: boxes, bags\\n\\n## BRANDS\\n• Miyake\\n• Tetsubin\\n• Kyusu\\n\\n## NOT SOLD\\n• Chopsticks, Spoons, Forks\\n• Glassware, Plasticware, Metalware', 'sitemap_analysis': 'Analysis unavailable', 'processing_metadata': {'processed_at': '2025-08-31 20:46:43', 'sitemaps_found': 3, 'urls_analyzed': 50, 'store_type': 'Shopify'}}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "\n",
    "with open('who_calls.jsonl', 'r') as f:\n",
    "    data = [json.loads(line) for line in f]\n",
    "\n",
    "print(data[0].keys())\n",
    "# query is the original user question\n",
    "print('query:', data[0]['query'])\n",
    "\n",
    "# prompt is the full prompt sent to the model\n",
    "print('prompt:', data[0]['prompt'])\n",
    "\n",
    "\n",
    "sample_size = 20\n",
    "data = np.random.choice(data, sample_size, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bbec65fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-09-18 12:29:49,839 - azure_oai - ERROR - error:151 - Missing required Azure OpenAI configuration\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python3\n",
    "\"\"\"\n",
    "Minimal Azure OpenAI Provider Example with clean_response - Copy & Paste Ready\n",
    "\"\"\"\n",
    "import asyncio\n",
    "import sys\n",
    "import os\n",
    "import json\n",
    "\n",
    "sys.path.insert(0, os.path.join('.', 'code', 'python'))\n",
    "\n",
    "from llm_providers.azure_oai import AzureOpenAIProvider\n",
    "\n",
    "async def test_azure_openai_completion(user_prompt, system_prompt, model=\"gpt-4.1-mini\"):\n",
    "    # Create provider (loads config automatically)\n",
    "    provider = AzureOpenAIProvider()\n",
    "    \n",
    "    # Get raw response from Azure OpenAI\n",
    "    client = provider.get_client()\n",
    "    \n",
    "    raw_response = await client.chat.completions.create(\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": system_prompt},\n",
    "            {\"role\": \"user\", \"content\": user_prompt}\n",
    "        ],\n",
    "        max_tokens=500,\n",
    "        temperature=0.8,\n",
    "        top_p=0.1,\n",
    "        stream=False,\n",
    "        presence_penalty=0.0,\n",
    "        frequency_penalty=0.0,\n",
    "        model=model\n",
    "    )\n",
    "    \n",
    "    # Get the raw content before cleaning\n",
    "    response = raw_response.choices[0].message.content\n",
    "    \n",
    "    return provider.clean_response(response)\n",
    "\n",
    "# asyncio.run(test_azure_openai_scoring())\n",
    "\n",
    "async def test_azure_openai_scoring(prompt, system_prompt=None, high_tier=True):\n",
    "    # Create provider (loads config automatically)\n",
    "    provider = AzureOpenAIProvider()\n",
    "    \n",
    "    schema = {\n",
    "        'score': 'integer between 0 and 100', \n",
    "        'description': 'short description of the item'\n",
    "    }\n",
    "    \n",
    "    # Get raw response from Azure OpenAI    \n",
    "    response = await provider.get_completion(\n",
    "        prompt=prompt,\n",
    "        schema=schema,\n",
    "        temperature=0.3,\n",
    "        max_tokens=500,\n",
    "        high_tier=high_tier,\n",
    "        system_prompt=system_prompt,\n",
    "    )\n",
    "    \n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2c434e27",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/20 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Missing required Azure OpenAI configuration",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mIndexError\u001b[39m                                Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 11\u001b[39m, in \u001b[36mget_mean_absolute_error\u001b[39m\u001b[34m(data, memory)\u001b[39m\n\u001b[32m     10\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m     score_with_high = \u001b[43mmemory\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\n\u001b[32m     12\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m:\n",
      "\u001b[31mIndexError\u001b[39m: list index out of range",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[31mValueError\u001b[39m                                Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 35\u001b[39m\n\u001b[32m     28\u001b[39m     \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mMAE when score is lower:\u001b[39m\u001b[33m\"\u001b[39m, error_lower / count_lower \u001b[38;5;28;01mif\u001b[39;00m count_lower > \u001b[32m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[32m0\u001b[39m)\n\u001b[32m     29\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m {\n\u001b[32m     30\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mtotal_mae\u001b[39m\u001b[33m\"\u001b[39m: error / \u001b[38;5;28mlen\u001b[39m(data),\n\u001b[32m     31\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mmae_higher\u001b[39m\u001b[33m\"\u001b[39m: error_higher / count_higher \u001b[38;5;28;01mif\u001b[39;00m count_higher > \u001b[32m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[32m0\u001b[39m,\n\u001b[32m     32\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mmae_lower\u001b[39m\u001b[33m\"\u001b[39m: error_lower / count_lower \u001b[38;5;28;01mif\u001b[39;00m count_lower > \u001b[32m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[32m0\u001b[39m\n\u001b[32m     33\u001b[39m     }, memory\n\u001b[32m---> \u001b[39m\u001b[32m35\u001b[39m stats, memory = \u001b[38;5;28;01mawait\u001b[39;00m get_mean_absolute_error(data)\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 13\u001b[39m, in \u001b[36mget_mean_absolute_error\u001b[39m\u001b[34m(data, memory)\u001b[39m\n\u001b[32m     11\u001b[39m     score_with_high = memory[idx]\n\u001b[32m     12\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m     score_with_high = \u001b[38;5;28;01mawait\u001b[39;00m test_azure_openai_scoring(prompt=item[\u001b[33m'\u001b[39m\u001b[33mprompt\u001b[39m\u001b[33m'\u001b[39m], high_tier=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m     14\u001b[39m     memory.append(score_with_high)\n\u001b[32m     16\u001b[39m new_score = score_with_high[\u001b[33m'\u001b[39m\u001b[33mscore\u001b[39m\u001b[33m'\u001b[39m]\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 52\u001b[39m, in \u001b[36mtest_azure_openai_scoring\u001b[39m\u001b[34m(prompt, system_prompt, high_tier)\u001b[39m\n\u001b[32m     46\u001b[39m schema = {\n\u001b[32m     47\u001b[39m     \u001b[33m'\u001b[39m\u001b[33mscore\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33minteger between 0 and 100\u001b[39m\u001b[33m'\u001b[39m, \n\u001b[32m     48\u001b[39m     \u001b[33m'\u001b[39m\u001b[33mdescription\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33mshort description of the item\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m     49\u001b[39m }\n\u001b[32m     51\u001b[39m \u001b[38;5;66;03m# Get raw response from Azure OpenAI    \u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m52\u001b[39m response = \u001b[38;5;28;01mawait\u001b[39;00m provider.get_completion(\n\u001b[32m     53\u001b[39m     prompt=prompt,\n\u001b[32m     54\u001b[39m     schema=schema,\n\u001b[32m     55\u001b[39m     temperature=\u001b[32m0.3\u001b[39m,\n\u001b[32m     56\u001b[39m     max_tokens=\u001b[32m500\u001b[39m,\n\u001b[32m     57\u001b[39m     high_tier=high_tier,\n\u001b[32m     58\u001b[39m     system_prompt=system_prompt,\n\u001b[32m     59\u001b[39m )\n\u001b[32m     61\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/code/conv/code/python/llm_providers/azure_oai.py:185\u001b[39m, in \u001b[36mAzureOpenAIProvider.get_completion\u001b[39m\u001b[34m(self, prompt, schema, model, temperature, max_tokens, timeout, high_tier, **kwargs)\u001b[39m\n\u001b[32m    182\u001b[39m \u001b[38;5;66;03m# Use specified model or get from config based on tier\u001b[39;00m\n\u001b[32m    183\u001b[39m model_to_use = model \u001b[38;5;28;01mif\u001b[39;00m model \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m.get_model_from_config(high_tier)\n\u001b[32m--> \u001b[39m\u001b[32m185\u001b[39m client = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mget_client\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    186\u001b[39m system_prompt = \u001b[33mf\u001b[39m\u001b[33m\"\"\"\u001b[39m\u001b[33mProvide a response that matches this JSON schema: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mjson.dumps(schema)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\"\"\u001b[39m\n\u001b[32m    188\u001b[39m logger.debug(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mSending completion request to Azure OpenAI with model: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_to_use\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/code/conv/code/python/llm_providers/azure_oai.py:87\u001b[39m, in \u001b[36mAzureOpenAIProvider.get_client\u001b[39m\u001b[34m(cls)\u001b[39m\n\u001b[32m     85\u001b[39m     error_msg = \u001b[33m\"\u001b[39m\u001b[33mMissing required Azure OpenAI configuration\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m     86\u001b[39m     logger.error(error_msg)\n\u001b[32m---> \u001b[39m\u001b[32m87\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(error_msg)\n\u001b[32m     89\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m     90\u001b[39m     \u001b[38;5;28mcls\u001b[39m._client = AsyncAzureOpenAI(\n\u001b[32m     91\u001b[39m         azure_endpoint=endpoint,\n\u001b[32m     92\u001b[39m         api_key=api_key,\n\u001b[32m     93\u001b[39m         api_version=api_version,\n\u001b[32m     94\u001b[39m         timeout=\u001b[32m30.0\u001b[39m  \u001b[38;5;66;03m# Set timeout explicitly\u001b[39;00m\n\u001b[32m     95\u001b[39m     )\n",
      "\u001b[31mValueError\u001b[39m: Missing required Azure OpenAI configuration"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "async def get_mean_absolute_error(data, memory=None):\n",
    "    error, error_higher, error_lower = 0, 0, 0\n",
    "    count_higher, count_lower = 0, 0\n",
    "    if memory is None:\n",
    "        memory = []\n",
    "\n",
    "    for idx, item in enumerate(tqdm(data)):\n",
    "        try:\n",
    "            score_with_high = memory[idx]\n",
    "        except:\n",
    "            score_with_high = await test_azure_openai_scoring(prompt=item['prompt'], high_tier=True)\n",
    "            memory.append(score_with_high)\n",
    "        \n",
    "        new_score = score_with_high['score']\n",
    "        print('new_score:', new_score, 'old_score:', item['ranking']['score'])\n",
    "        error += abs(new_score - item['ranking']['score'])\n",
    "        if new_score > item['ranking']['score']:\n",
    "            error_lower += abs(new_score - item['ranking']['score'])\n",
    "            count_lower += 1\n",
    "        else:\n",
    "            error_higher += abs(new_score - item['ranking']['score'])\n",
    "            count_higher += 1\n",
    "\n",
    "    print(\"Total MAE:\", error / len(data))\n",
    "    print(\"MAE when score is higher:\", error_higher / count_higher if count_higher > 0 else 0)\n",
    "    print(\"MAE when score is lower:\", error_lower / count_lower if count_lower > 0 else 0)\n",
    "    return {\n",
    "        \"total_mae\": error / len(data),\n",
    "        \"mae_higher\": error_higher / count_higher if count_higher > 0 else 0,\n",
    "        \"mae_lower\": error_lower / count_lower if count_lower > 0 else 0\n",
    "    }, memory\n",
    "\n",
    "stats, memory = await get_mean_absolute_error(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a098d810",
   "metadata": {},
   "outputs": [],
   "source": [
    "async def regenerate_responses_low_tier(data, system_prompt):\n",
    "    new_data = []\n",
    "    for idx, item in enumerate(tqdm(data)):\n",
    "        score_low = await test_azure_openai_scoring(prompt=item['prompt'], system_prompt=system_prompt, high_tier=False)\n",
    "        # copy all elements from item and add score_low\n",
    "        new_item = item.copy()\n",
    "        new_item['ranking'] = score_low\n",
    "        new_data.append(new_item)\n",
    "    \n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b00cb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score decreased from 75 to 30\n",
      "prompt: Assign a score between 0 and 100 to the following site based \n",
      "        the likelihood that the site will contain an answer to the user's question.\n",
      "        If the user is looking to buy a product, the site should sell the product, not \n",
      "        just have useful information. \n",
      "\n",
      "The user's question is: I am interested in making gluten free bread. What kind of special equipment do I need?\n",
      "\n",
      "The site's description is: {'url': 'anson-mills.myshopify.com', '@type': 'Shopify', 'name': 'Anson Mills', 'category': 'Flour & Grains', 'description': 'Heritage grains and heirloom varieties from historic Southern mill', 'extended_description': 'Heritage grains and heirloom varieties from historic Southern mill. Anson Mills typically offers stone‑milled flours; heritage grains; baking mixes; starters & education; and whole grains. Highlights milling dates, protein % and recommended formulas for breads & pastry. Seasonal & limited releases may include limited heritage lots; holiday cookie & pie mixes.', 'notable_products': ['flour', 'heritage grains'], 'detailed_description': \"## PRODUCTS BY CATEGORY\\n• Grits: antebellum coarse white grits, antebellum coarse yellow grits, colonial pencil cob grits, quick grits, blue corn grits\\n• Cornmeal: white cornmeal (fine, coarse), yellow cornmeal (fine, coarse), blue cornmeal\\n• Polenta: fine white polenta, fine yellow polenta, rustic coarse polenta, rice-based polenta\\n• Rice: Carolina Gold rice, Charleston Gold rice, rice grits, rice flour\\n• Wheat Flours: whole grain wheat flour, graham flour, pastry flour, cake flour, bread flour, flatbread flour, pizza flour, pasta makers' flour\\n• Rye & Buckwheat: heirloom rye flour, sobakoh, buckwheat flour\\n• Oats: stone-cut oats, toasted oat flour\\n• Farro: farro piccolo, farro medio, slow-roasted farro\\n• Legumes & Seeds: sea island red peas, benne seeds, bennecake flour\\n• Other: popping corn, emmer semolina, gluten-free flour\\n\\n## ALL PRODUCTS A-Z\\n• Antebellum coarse white grits (1lb, 2lb)\\n• Antebellum coarse yellow grits (1lb, 2lb)\\n• Benne seeds (8oz)\\n• Bennecake flour (1lb)\\n• Blue corn grits (1lb)\\n• Blue cornmeal (1lb)\\n• Bread flour (2lb)\\n• Buckwheat flour (1lb)\\n• Cake flour (2lb)\\n• Carolina Gold rice (1lb, 2lb, 10lb)\\n• Charleston Gold rice (1lb, 2lb)\\n• Colonial pencil cob grits (1lb)\\n• Emmer semolina (1lb)\\n• Farro medio (1lb)\\n• Farro piccolo (1lb)\\n• Flatbread flour (2lb)\\n• Gluten-free flour (1lb)\\n• Graham flour (2lb)\\n• Heirloom rye flour (1lb)\\n• Pasta makers' flour (2lb)\\n• Pastry flour (2lb)\\n• Popping corn (1lb)\\n• Quick grits (1lb)\\n• Rice flour (1lb)\\n• Rice grits (1lb)\\n• Rustic coarse polenta (1lb)\\n• Sea island red peas (1lb)\\n• Slow-roasted farro (1lb)\\n• Sobakoh (1lb)\\n• Stone-cut oats (1lb)\\n• Toasted oat flour (1lb)\\n• White cornmeal (fine, coarse) (1lb)\\n• Yellow cornmeal (fine, coarse) (1lb)\\n• Fine white polenta (1lb)\\n• Fine yellow polenta (1lb)\\n\\n## SPECIFICATIONS\\n• Sizes: 8oz, 1lb, 2lb, 10lb\\n• Flavors: white, yellow, blue, rice, rye, buckwheat, emmer, benne\\n• Colors: white, yellow, blue\\n• Materials: corn, rice, wheat, rye, buckwheat, oats, farro, peas, benne, emmer\\n• Package types: bags, boxes\\n\\n## BRANDS\\n• Anson Mills\\n\\n## NOT SOLD\\n• Spices, oils, vinegars\\n• Dairy, eggs, meat\\n• Fruits, vegetables, nuts\\n• Sauces, jams, honey\\n• Coffee, tea, beverages\", 'sitemap_analysis': \"1. Product Categories and Types:\\nThe URLs indicate that Anson Mills specializes in a variety of heritage grains and grain-based products. The main product categories include:\\n- Grits: antebellum coarse white/yellow, colonial pencil cob, quick grits, blue corn grits\\n- Cornmeal: white, yellow, fine, coarse, blue cornmeal\\n- Polenta: fine white/yellow, rustic coarse, rice-based polenta\\n- Rice: Carolina Gold, Charleston Gold, rice grits, rice flour\\n- Wheat Flours: whole grain, graham, pastry, cake, bread, flatbread, pizza, pasta makers' flour\\n- Rye and Buckwheat: heirloom rye flour, sobakoh, buckwheat flour\\n- Oats: stone-cut oats, toasted oat flour\\n- Farro: farro piccolo, farro medio, slow-roasted farro\\n- Legumes and Seeds: sea island red peas, benne seeds, bennecake flour\\n- Other: popping corn, emmer semolina, gluten-free flour\\n\\n2. Site Structure and Organization:\\nThe URLs follow a clear and standard Shopify structure, with the homepage at '/', and individual products under '/products/'. Each product has a descriptive, keyword-rich slug, which aids in SEO and user navigation. There is no evidence of subcategories or nested collections in the URL structure, suggesting that products are likely organized into collections or categories on the front end, but not reflected in the URL path. The site likely uses Shopify's collection features to group products by type (e.g., grits, flours, rice, polenta).\\n\\n3. Special Collections or Features:\\nSeveral products reference historical or regional varieties (e.g., 'antebellum', 'colonial', 'Carolina Gold', 'Charleston Gold', 'thirteen colony', 'artisan', 'native', 'heirloom'), indicating a focus on heritage grains and traditional milling techniques. The presence of gluten-free flour and specialty flours for pizza, pasta, and flatbread suggests attention to diverse culinary needs. The use of terms like 'artisan', 'handmade', and 'ancient' points to premium, small-batch, or craft production. There may be special collections centered around heritage grains, gluten-free options, or products for specific culinary uses (baking, pasta making, etc.).\\n\\n4. Target Audience Based on URL Patterns:\\nThe product naming and variety suggest the target audience includes:\\n- Culinary professionals (chefs, bakers, restaurateurs) seeking specialty and heritage grains\\n- Home cooks and food enthusiasts interested in traditional, artisanal, and regional ingredients\\n- Health-conscious consumers looking for whole grain, gluten-free, and non-GMO options\\n- Individuals interested in Southern, colonial, or historical American cuisine\\n- Possibly specialty retailers or small food businesses sourcing premium ingredients\\n\\nOverall, Anson Mills appears to be a specialty store focused on high-quality, heritage grains and flours, catering to both professional and passionate home cooks who value tradition, authenticity, and culinary excellence.\", 'processing_metadata': {'processed_at': '2025-08-31 12:35:12', 'sitemaps_found': 3, 'urls_analyzed': 46, 'store_type': 'Shopify'}}\n",
      "\n",
      "gpt-4.1-mini response: {'score': 75, 'description': 'The site sells gluten-free flour and other specialty flours, which is relevant for making gluten-free bread. While it primarily sells products rather than detailed equipment guides, it also offers starters and education, suggesting some useful information on equipment might be available.', 'query': 'gluten free bread special equipment'}\n",
      "gpt-4.1 response: {'score': 30, 'description': 'The site sells gluten-free flour and heritage grains, but does not appear to sell special equipment for making gluten-free bread. It may offer some educational content, but is primarily a source for ingredients.'}\n",
      "----------------------------------------\n",
      "Analysis response: system prompt is too general and does not require the model to break down the user's question, model may focus on keyword matches (e.g., 'gluten-free flour') rather than deeper relevance, model may not distinguish between selling ingredients and providing information about equipment, model may not be prompted to consider whether the site answers the specific aspect of the question (special equipment), lack of instruction to analyze the completeness of the answer, no requirement to explain reasoning for the score\n",
      "New system prompt: Provide a response that matches this JSON schema: {\"score\": \"integer between 0 and 100\", \"description\": \"short description of the item\"}. In addition to assigning a score, carefully analyze the user's question to identify its key requirements. Break down whether the site directly addresses all aspects of the question, including any specific needs or sub-questions. Consider if the site provides a complete and direct answer, or only partial or tangential information. Justify your score by explaining which parts of the user's question are fully, partially, or not addressed by the site.\n",
      "new_score: 30\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:38<00:00,  1.94s/it]\n",
      "100%|██████████| 20/20 [00:00<00:00, 25474.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new_score: 30 old_score: 20\n",
      "new_score: 40 old_score: 30\n",
      "new_score: 30 old_score: 20\n",
      "new_score: 85 old_score: 80\n",
      "new_score: 10 old_score: 10\n",
      "new_score: 30 old_score: 30\n",
      "new_score: 30 old_score: 40\n",
      "new_score: 10 old_score: 10\n",
      "new_score: 90 old_score: 85\n",
      "new_score: 10 old_score: 30\n",
      "new_score: 85 old_score: 90\n",
      "new_score: 80 old_score: 40\n",
      "new_score: 30 old_score: 30\n",
      "new_score: 85 old_score: 85\n",
      "new_score: 20 old_score: 30\n",
      "new_score: 60 old_score: 40\n",
      "new_score: 90 old_score: 90\n",
      "new_score: 20 old_score: 10\n",
      "new_score: 20 old_score: 10\n",
      "new_score: 90 old_score: 90\n",
      "Total MAE: 8.25\n",
      "MAE when score is higher: 4.090909090909091\n",
      "MAE when score is lower: 13.333333333333334\n",
      "Score decreased from 90 to 85\n",
      "prompt: Assign a score between 0 and 100 to the following site based \n",
      "        the likelihood that the site will contain an answer to the user's question.\n",
      "        If the user is looking to buy a product, the site should sell the product, not \n",
      "        just have useful information. \n",
      "\n",
      "The user's question is: I am interested in chai with interesting spices.\n",
      "\n",
      "The site's description is: {'url': 'burlapandbarrel.com', '@type': 'Shopify', 'name': 'Burlap and Barrel', 'category': 'Spices', 'description': 'Single origin spices with farmer partnerships', 'detailed_description': \"## PRODUCTS BY CATEGORY\\n• Single-Origin Spices: ground cinnamon, wild mountain cumin, silk chili flakes, sweet allspice, black urfa chili, king caraway, cardamom, cloves, saffron, sumac, turmeric, peppercorns, paprika, hibiscus, mint, oregano, garlic, wild ramps, wild pompona vanilla, fermented locust beans\\n• Spice Blends & Collections: chef's collection, fundamentals collection, complete collection, cinnamon collection, peppercorn collection, Floyd Cardoz masalas, Anjali's chai masala\\n• Collaborative & Limited Edition: autumn spice almond butter collaboration\\n• Merchandise & Accessories: kitchen towels, hats, empty spice jars\\n\\n## ALL PRODUCTS A-Z\\n• Anjali's chai masala (jar)\\n• Autumn spice almond butter collaboration (jar)\\n• Black urfa chili (jar, bulk)\\n• Cardamom (jar, bulk)\\n• Chef's collection (box)\\n• Cinnamon collection (box)\\n• Cloves (jar, bulk)\\n• Complete collection (box)\\n• Empty spice jars (glass)\\n• Fermented locust beans (jar)\\n• Floyd Cardoz masalas (jar)\\n• Fundamentals collection (box)\\n• Garlic (jar, bulk)\\n• Ground cinnamon (jar, bulk)\\n• Hats (black, white)\\n• Hibiscus (jar, bulk)\\n• King caraway (jar, bulk)\\n• Kitchen towels (cotton)\\n• Mint (jar, bulk)\\n• Oregano (jar, bulk)\\n• Paprika (jar, bulk)\\n• Peppercorn collection (box)\\n• Peppercorns (jar, bulk)\\n• Saffron (jar, bulk)\\n• Silk chili flakes (jar, bulk)\\n• Sumac (jar, bulk)\\n• Sweet allspice (jar, bulk)\\n• Turmeric (jar, bulk)\\n• Wild mountain cumin (jar, bulk)\\n• Wild pompona vanilla (jar)\\n• Wild ramps (jar)\\n\\n## SPECIFICATIONS\\n• Sizes: jar, bulk, box\\n• Flavors: cinnamon, cumin, chili, allspice, cardamom, cloves, saffron, sumac, turmeric, peppercorn, paprika, hibiscus, mint, oregano, garlic, caraway, vanilla, locust bean, ramps, masala, chai, almond butter\\n• Colors: black, white (hats)\\n• Materials: glass (jars), cotton (towels)\\n• Package types: jars, boxes, bags\\n\\n## BRANDS\\n• Burlap & Barrel\\n\\n## NOT SOLD\\n• Dairy, meat, seafood\\n• Fresh produce, grains, pasta\\n• Beverages, snacks, sweets\\n• Cookware, utensils, electronics\\n• Personal care, cleaning products\", 'sitemap_analysis': \"Based on the provided URLs from the Burlap and Barrel Shopify store, several insights can be drawn:\\n\\n1. Product Categories and Types:\\n- The majority of URLs follow the pattern '/products/[product-name]', indicating a product-focused site structure.\\n- Products are primarily single-origin spices and seasonings, such as ground cinnamon, wild mountain cumin, silk chili flakes, sweet allspice, black urfa chili, king caraway, cardamom, cloves, saffron, sumac, turmeric, peppercorns, paprika, hibiscus, mint, oregano, garlic, and more.\\n- There are also spice blends and collections (e.g., chef's collection, fundamentals collection, complete collection, cinnamon collection, peppercorn collection, Floyd Cardoz masalas, Anjali's chai masala).\\n- Some non-spice products are present, such as kitchen towels, hats, empty spice jars, and almond butter collaboration, indicating a small selection of branded merchandise and collaborative products.\\n- Specialty items like wild ramps, wild pompona vanilla, and fermented locust beans suggest a focus on rare or unique ingredients.\\n\\n2. Site Structure and Organization:\\n- The homepage is at the root URL ('/'), while all products are under '/products/'.\\n- Collections and bundles are treated as products, not as separate collection pages, which is typical for Shopify stores that use product pages for curated sets.\\n- The URL structure is clean and descriptive, using hyphenated product names for SEO and clarity.\\n- There is no evidence of subcategories or nested collections in the URL structure; all products are listed at the same level.\\n\\n3. Special Collections or Features:\\n- Multiple curated collections are offered, such as chef's collection, fundamentals collection, complete collection, cinnamon collection, and peppercorn collection. These likely bundle several spices for convenience or gifting.\\n- Collaborative products (e.g., autumn spice almond butter collaboration) and limited-edition blends (e.g., Floyd Cardoz masalas, Anjali's chai masala) suggest partnerships with chefs or other brands.\\n- The 'spice club' product indicates a subscription or membership feature, allowing customers to receive spices regularly.\\n- Accessories like empty spice jars and kitchen towels support the main product line and encourage repeat purchases.\\n\\n4. Target Audience Based on URL Patterns:\\n- The focus on single-origin, rare, and specialty spices, as well as chef collaborations, suggests the target audience includes home cooks, food enthusiasts, and professional chefs seeking high-quality, unique ingredients.\\n- The presence of curated collections and a subscription club appeals to gift shoppers and those interested in culinary exploration.\\n- The clean, descriptive URLs and lack of subcategories indicate a user-friendly site aimed at a broad audience, but with a particular appeal to those who value transparency, provenance, and culinary adventure.\\n\\nOverall, Burlap and Barrel's store is organized around a wide variety of single-origin spices, curated collections, and specialty collaborations, with a straightforward site structure that makes it easy for customers to browse and purchase unique ingredients. The target audience is likely food lovers, adventurous cooks, and professionals seeking high-quality, ethically sourced spices.\", 'processing_metadata': {'processed_at': '2025-08-31 12:47:11', 'sitemaps_found': 3, 'urls_analyzed': 50, 'store_type': 'Shopify'}}\n",
      "\n",
      "gpt-4.1-mini response: {'score': 90, 'description': \"The site sells a variety of single-origin spices and spice blends, including 'Anjali's chai masala,' which directly relates to chai with interesting spices. It offers both individual spices and curated chai masala blends, meeting the user's interest in chai with unique spices. The site is a Shopify store focused on selling these products, not just providing information, which aligns well with the user's likely intent to purchase. The only minor limitation is that the site does not explicitly mention ready-to-drink chai beverages, but since the user asked about chai with interesting spices (likely for making chai), the site fully addresses the key requirement.\"}\n",
      "gpt-4.1 response: {'score': 85, 'description': 'Burlap and Barrel sells a wide variety of single-origin spices and spice blends, including chai masala and other interesting spices suitable for making chai. The site is product-focused and allows direct purchase of these items.'}\n",
      "----------------------------------------\n",
      "Analysis response: the system prompt may not sufficiently emphasize the need to distinguish between direct and indirect relevance, the model might conflate related products (spices) with the specific user intent (chai with interesting spices), the prompt does not explicitly require the model to check if the site sells the exact item sought or just components, the model may overvalue partial matches (spice blends, chai masala) without considering completeness, lack of instruction to penalize for missing direct product offerings, insufficient guidance to weigh tangential information lower\n",
      "New system prompt: Provide a response that matches this JSON schema: {\"score\": \"integer between 0 and 100\", \"description\": \"short description of the item\"}. In addition to assigning a score, carefully analyze the user's question to identify its key requirements. Break down whether the site directly addresses all aspects of the question, including any specific needs or sub-questions. Consider if the site provides a complete and direct answer, or only partial or tangential information. Justify your score by explaining which parts of the user's question are fully, partially, or not addressed by the site. Additionally, explicitly distinguish between sites that offer the exact item or solution sought and those that only provide related or component items. Penalize scores for sites that do not directly fulfill the user's main intent, and weigh tangential or partial matches lower. Ensure your analysis clearly separates direct relevance from indirect or related information.\n",
      "new_score: 90\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:41<00:00,  2.07s/it]\n",
      "100%|██████████| 20/20 [00:00<00:00, 25183.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new_score: 30 old_score: 20\n",
      "new_score: 40 old_score: 40\n",
      "new_score: 30 old_score: 20\n",
      "new_score: 85 old_score: 70\n",
      "new_score: 10 old_score: 10\n",
      "new_score: 30 old_score: 20\n",
      "new_score: 30 old_score: 40\n",
      "new_score: 10 old_score: 10\n",
      "new_score: 90 old_score: 85\n",
      "new_score: 10 old_score: 20\n",
      "new_score: 85 old_score: 90\n",
      "new_score: 80 old_score: 40\n",
      "new_score: 30 old_score: 30\n",
      "new_score: 85 old_score: 75\n",
      "new_score: 20 old_score: 30\n",
      "new_score: 60 old_score: 20\n",
      "new_score: 90 old_score: 90\n",
      "new_score: 20 old_score: 10\n",
      "new_score: 20 old_score: 10\n",
      "new_score: 90 old_score: 90\n",
      "Total MAE: 9.75\n",
      "MAE when score is higher: 3.5\n",
      "MAE when score is lower: 16.0\n",
      "Score decreased from 90 to 85\n",
      "prompt: Assign a score between 0 and 100 to the following site based \n",
      "        the likelihood that the site will contain an answer to the user's question.\n",
      "        If the user is looking to buy a product, the site should sell the product, not \n",
      "        just have useful information. \n",
      "\n",
      "The user's question is: I am interested in chai with interesting spices.\n",
      "\n",
      "The site's description is: {'url': 'burlapandbarrel.com', '@type': 'Shopify', 'name': 'Burlap and Barrel', 'category': 'Spices', 'description': 'Single origin spices with farmer partnerships', 'detailed_description': \"## PRODUCTS BY CATEGORY\\n• Single-Origin Spices: ground cinnamon, wild mountain cumin, silk chili flakes, sweet allspice, black urfa chili, king caraway, cardamom, cloves, saffron, sumac, turmeric, peppercorns, paprika, hibiscus, mint, oregano, garlic, wild ramps, wild pompona vanilla, fermented locust beans\\n• Spice Blends & Collections: chef's collection, fundamentals collection, complete collection, cinnamon collection, peppercorn collection, Floyd Cardoz masalas, Anjali's chai masala\\n• Collaborative & Limited Edition: autumn spice almond butter collaboration\\n• Merchandise & Accessories: kitchen towels, hats, empty spice jars\\n\\n## ALL PRODUCTS A-Z\\n• Anjali's chai masala (jar)\\n• Autumn spice almond butter collaboration (jar)\\n• Black urfa chili (jar, bulk)\\n• Cardamom (jar, bulk)\\n• Chef's collection (box)\\n• Cinnamon collection (box)\\n• Cloves (jar, bulk)\\n• Complete collection (box)\\n• Empty spice jars (glass)\\n• Fermented locust beans (jar)\\n• Floyd Cardoz masalas (jar)\\n• Fundamentals collection (box)\\n• Garlic (jar, bulk)\\n• Ground cinnamon (jar, bulk)\\n• Hats (black, white)\\n• Hibiscus (jar, bulk)\\n• King caraway (jar, bulk)\\n• Kitchen towels (cotton)\\n• Mint (jar, bulk)\\n• Oregano (jar, bulk)\\n• Paprika (jar, bulk)\\n• Peppercorn collection (box)\\n• Peppercorns (jar, bulk)\\n• Saffron (jar, bulk)\\n• Silk chili flakes (jar, bulk)\\n• Sumac (jar, bulk)\\n• Sweet allspice (jar, bulk)\\n• Turmeric (jar, bulk)\\n• Wild mountain cumin (jar, bulk)\\n• Wild pompona vanilla (jar)\\n• Wild ramps (jar)\\n\\n## SPECIFICATIONS\\n• Sizes: jar, bulk, box\\n• Flavors: cinnamon, cumin, chili, allspice, cardamom, cloves, saffron, sumac, turmeric, peppercorn, paprika, hibiscus, mint, oregano, garlic, caraway, vanilla, locust bean, ramps, masala, chai, almond butter\\n• Colors: black, white (hats)\\n• Materials: glass (jars), cotton (towels)\\n• Package types: jars, boxes, bags\\n\\n## BRANDS\\n• Burlap & Barrel\\n\\n## NOT SOLD\\n• Dairy, meat, seafood\\n• Fresh produce, grains, pasta\\n• Beverages, snacks, sweets\\n• Cookware, utensils, electronics\\n• Personal care, cleaning products\", 'sitemap_analysis': \"Based on the provided URLs from the Burlap and Barrel Shopify store, several insights can be drawn:\\n\\n1. Product Categories and Types:\\n- The majority of URLs follow the pattern '/products/[product-name]', indicating a product-focused site structure.\\n- Products are primarily single-origin spices and seasonings, such as ground cinnamon, wild mountain cumin, silk chili flakes, sweet allspice, black urfa chili, king caraway, cardamom, cloves, saffron, sumac, turmeric, peppercorns, paprika, hibiscus, mint, oregano, garlic, and more.\\n- There are also spice blends and collections (e.g., chef's collection, fundamentals collection, complete collection, cinnamon collection, peppercorn collection, Floyd Cardoz masalas, Anjali's chai masala).\\n- Some non-spice products are present, such as kitchen towels, hats, empty spice jars, and almond butter collaboration, indicating a small selection of branded merchandise and collaborative products.\\n- Specialty items like wild ramps, wild pompona vanilla, and fermented locust beans suggest a focus on rare or unique ingredients.\\n\\n2. Site Structure and Organization:\\n- The homepage is at the root URL ('/'), while all products are under '/products/'.\\n- Collections and bundles are treated as products, not as separate collection pages, which is typical for Shopify stores that use product pages for curated sets.\\n- The URL structure is clean and descriptive, using hyphenated product names for SEO and clarity.\\n- There is no evidence of subcategories or nested collections in the URL structure; all products are listed at the same level.\\n\\n3. Special Collections or Features:\\n- Multiple curated collections are offered, such as chef's collection, fundamentals collection, complete collection, cinnamon collection, and peppercorn collection. These likely bundle several spices for convenience or gifting.\\n- Collaborative products (e.g., autumn spice almond butter collaboration) and limited-edition blends (e.g., Floyd Cardoz masalas, Anjali's chai masala) suggest partnerships with chefs or other brands.\\n- The 'spice club' product indicates a subscription or membership feature, allowing customers to receive spices regularly.\\n- Accessories like empty spice jars and kitchen towels support the main product line and encourage repeat purchases.\\n\\n4. Target Audience Based on URL Patterns:\\n- The focus on single-origin, rare, and specialty spices, as well as chef collaborations, suggests the target audience includes home cooks, food enthusiasts, and professional chefs seeking high-quality, unique ingredients.\\n- The presence of curated collections and a subscription club appeals to gift shoppers and those interested in culinary exploration.\\n- The clean, descriptive URLs and lack of subcategories indicate a user-friendly site aimed at a broad audience, but with a particular appeal to those who value transparency, provenance, and culinary adventure.\\n\\nOverall, Burlap and Barrel's store is organized around a wide variety of single-origin spices, curated collections, and specialty collaborations, with a straightforward site structure that makes it easy for customers to browse and purchase unique ingredients. The target audience is likely food lovers, adventurous cooks, and professionals seeking high-quality, ethically sourced spices.\", 'processing_metadata': {'processed_at': '2025-08-31 12:47:11', 'sitemaps_found': 3, 'urls_analyzed': 50, 'store_type': 'Shopify'}}\n",
      "\n",
      "gpt-4.1-mini response: {'score': 90, 'description': \"The site Burlap and Barrel directly sells spices and spice blends, including 'Anjali's chai masala,' which is a chai spice blend with interesting spices. This matches the user's interest in chai with interesting spices very well. The site offers both single-origin spices and curated spice blends, providing a complete solution rather than just information. The presence of a specific chai masala product indicates a direct match rather than a tangential or partial one. The only minor deduction is that the site does not explicitly mention ready-to-drink chai beverages, but since the user asked for chai with interesting spices (which typically refers to the spice blend), the site fully addresses the main intent by selling the exact product sought.\"}\n",
      "gpt-4.1 response: {'score': 85, 'description': 'Burlap and Barrel sells a wide variety of single-origin spices and spice blends, including chai masala and other interesting spices suitable for making chai. The site is product-focused and allows direct purchase of these items.'}\n",
      "----------------------------------------\n",
      "Analysis response: The system prompt may not sufficiently emphasize the need to distinguish between sites that sell the exact item sought and those that only sell related components, the model may conflate selling ingredients (spices) with selling the final product (chai), the prompt may not require the model to explicitly check if the site offers the complete solution (chai with interesting spices) versus just ingredients, the model may be biased towards high scores for sites that are tangentially related (spice sellers) without penalizing for lack of direct product match, the prompt may not require a step-by-step mapping of user intent to site offerings, the model may not be prompted to consider the user's main goal (buying chai) versus secondary goals (finding spices)\n",
      "New system prompt: Provide a response that matches this JSON schema: {\"score\": \"integer between 0 and 100\", \"description\": \"short description of the item\"}. In addition to assigning a score, carefully analyze the user's question to identify its key requirements. Break down whether the site directly addresses all aspects of the question, including any specific needs or sub-questions. Consider if the site provides a complete and direct answer, or only partial or tangential information. Justify your score by explaining which parts of the user's question are fully, partially, or not addressed by the site. Additionally, explicitly distinguish between sites that offer the exact item or solution sought and those that only provide related or component items. Penalize scores for sites that do not directly fulfill the user's main intent, and weigh tangential or partial matches lower. Ensure your analysis clearly separates direct relevance from indirect or related information. Furthermore, explicitly verify whether the site offers the complete solution the user is seeking, rather than just related ingredients or components. If the site only provides components or information but not the full solution, assign a significantly lower score and clearly explain this distinction in your justification. Always map each key requirement from the user's question to the site's offerings and highlight any gaps or mismatches.\n",
      "new_score: 85\n",
      "================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:52<00:00,  2.63s/it]\n",
      "100%|██████████| 20/20 [00:00<00:00, 24549.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new_score: 30 old_score: 20\n",
      "new_score: 40 old_score: 20\n",
      "new_score: 30 old_score: 10\n",
      "new_score: 85 old_score: 60\n",
      "new_score: 10 old_score: 10\n",
      "new_score: 30 old_score: 20\n",
      "new_score: 30 old_score: 30\n",
      "new_score: 10 old_score: 10\n",
      "new_score: 90 old_score: 75\n",
      "new_score: 10 old_score: 20\n",
      "new_score: 85 old_score: 85\n",
      "new_score: 80 old_score: 30\n",
      "new_score: 30 old_score: 20\n",
      "new_score: 85 old_score: 75\n",
      "new_score: 20 old_score: 20\n",
      "new_score: 60 old_score: 30\n",
      "new_score: 90 old_score: 90\n",
      "new_score: 20 old_score: 10\n",
      "new_score: 20 old_score: 10\n",
      "new_score: 90 old_score: 85\n",
      "Total MAE: 11.75\n",
      "MAE when score is higher: 1.4285714285714286\n",
      "MAE when score is lower: 17.307692307692307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "async def improve_system_prompt(system_prompt, prompt):\n",
    "    schema = {\n",
    "                \"analysis\": \"comma separated list explaining why the small model might have given an overly optimistic score\",\n",
    "                \"new_system_prompt\": \"string with a new system prompt that could help the small model provide a more accurate score\"\n",
    "            }\n",
    "\n",
    "    new_prompt = f\"\"\"Here is a system prompt and query given to a small model to score a website for its relevance to a user question. The small model gave the site a high score, but a more advanced model disagreed and gave it a lower score. Analyze the system prompt, query, and the small model's response, and explain why the small model might have given an overly optimistic score. Only use general statements in your analysis, e.g., the system prompt needs to provoke the model to break down the query. Based on that analysis, suggest a new system prompt that together with the original user query could help the small model provide a more accurate score. \n",
    "Make sure the system prompt is general and does not have any details that is tied to the specific prompt. For example if the query is asking about a product, there should not be any mention of product in the system prompt.\n",
    "The new system prompt should be a superset of the original system prompt, i.e., it should include all the instructions from the original system prompt, but add more instructions to help the model provide a more accurate score.\n",
    "## System Prompt:\n",
    "{system_prompt}\n",
    "## Query:\n",
    "{prompt}\"\"\"\n",
    "    analysis_response = await test_azure_openai_completion(\n",
    "        user_prompt=new_prompt,\n",
    "        system_prompt=f\"You are an expert AI assistant that helps improve prompts for other AI models. Respond with the following json format: {json.dumps(schema)}\",\n",
    "        model=\"gpt-4.1\"\n",
    "    )\n",
    "    print('Analysis response:', analysis_response['analysis'])\n",
    "    print('New system prompt:', analysis_response['new_system_prompt'])\n",
    "    return analysis_response\n",
    "\n",
    "new_data = data.copy()\n",
    "new_stats = stats.copy()\n",
    "n_repeat = 0\n",
    "\n",
    "score_schema = {\n",
    "                    'score': 'integer between 0 and 100', \n",
    "                    'description': 'short description of the item'\n",
    "            }\n",
    "system_prompt = f\"\"\"Provide a response that matches this JSON schema: {json.dumps(score_schema)}\"\"\"\n",
    "\n",
    "while new_stats['total_mae'] >= 5 and n_repeat < 3:\n",
    "    # find everything with score above 70 and print the function name and score\n",
    "    for idx, item in enumerate(new_data):\n",
    "        if item['ranking']['score'] > 70:\n",
    "            score_high = memory[idx]\n",
    "            if score_high['score'] < item['ranking']['score']:\n",
    "                print(f\"Score decreased from {item['ranking']['score']} to {score_high['score']}\")\n",
    "                print(f\"prompt: {item['prompt']}\")\n",
    "                print('gpt-4.1-mini response:', item['ranking'])\n",
    "                print('gpt-4.1 response:', score_high)\n",
    "                print('-' * 40)\n",
    "\n",
    "                \n",
    "                # improve the system prompt\n",
    "                analysis_response = await improve_system_prompt(system_prompt, item['prompt'])\n",
    "                system_prompt = analysis_response['new_system_prompt']\n",
    "                \n",
    "                scoring_low = await test_azure_openai_scoring(prompt=item['prompt'], system_prompt=system_prompt, high_tier=False)\n",
    "                print('new_score:', scoring_low['score'])\n",
    "                print('=' * 80)\n",
    "                break\n",
    "    new_data = await regenerate_responses_low_tier(new_data, system_prompt)\n",
    "    new_stats, _ = await get_mean_absolute_error(new_data, memory)\n",
    "    n_repeat += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9c914de",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "3.12.7",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
